{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!unzip 基于用户画像的商品推荐挑战赛2.0公开数据-初赛.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:54:35.664021Z",
     "iopub.status.busy": "2023-08-29T03:54:35.663128Z",
     "iopub.status.idle": "2023-08-29T03:54:52.174212Z",
     "shell.execute_reply": "2023-08-29T03:54:52.173314Z",
     "shell.execute_reply.started": "2023-08-29T03:54:35.663992Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "train_data = pd.read_csv(\"基于用户画像的商品推荐挑战赛2.0公开数据-初赛/train.csv\")\n",
    "test_data = pd.read_csv(\"基于用户画像的商品推荐挑战赛2.0公开数据-初赛/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:54:52.194654Z",
     "iopub.status.busy": "2023-08-29T03:54:52.194332Z",
     "iopub.status.idle": "2023-08-29T03:54:52.205521Z",
     "shell.execute_reply": "2023-08-29T03:54:52.204780Z",
     "shell.execute_reply.started": "2023-08-29T03:54:52.194630Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>label</th>\n",
       "      <th>brand</th>\n",
       "      <th>model</th>\n",
       "      <th>province</th>\n",
       "      <th>city</th>\n",
       "      <th>enum_tag</th>\n",
       "      <th>numeric_tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ac+1xFYOjvFPT6sUKOPjaA==</td>\n",
       "      <td>0</td>\n",
       "      <td>HUAWEI</td>\n",
       "      <td>HUAWEI NOVA 7 5G</td>\n",
       "      <td>河北</td>\n",
       "      <td>邯郸</td>\n",
       "      <td>tagid:4dbc6e1a685a4dfdf9d58c37f2d53c52;time:20...</td>\n",
       "      <td>tagid:c03968985782bb457f8124b244c50c30;value:1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9RQtNBN4njiXfA60ZbH3Sw==</td>\n",
       "      <td>0</td>\n",
       "      <td>unkown</td>\n",
       "      <td>unkown</td>\n",
       "      <td>黑龙江</td>\n",
       "      <td>齐齐哈尔</td>\n",
       "      <td>unkown</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6x3FhIia6hm3tZTvPPxuLA==</td>\n",
       "      <td>0</td>\n",
       "      <td>OPPO</td>\n",
       "      <td>OPPO RENO6 5G</td>\n",
       "      <td>广东</td>\n",
       "      <td>茂名</td>\n",
       "      <td>unkown</td>\n",
       "      <td>tagid:d584da8a6f878c250073a365cf5ef73e;value:9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>zHOsTImR/JRdOYHSn1aAMg==</td>\n",
       "      <td>1</td>\n",
       "      <td>VIVO</td>\n",
       "      <td>VIVO Y72T</td>\n",
       "      <td>内蒙古</td>\n",
       "      <td>巴彦淖尔</td>\n",
       "      <td>unkown</td>\n",
       "      <td>tagid:ecdf085acb4362af9772772f295ea000;value:0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rULwSixw+GLkr1fy/mvMpg==</td>\n",
       "      <td>0</td>\n",
       "      <td>VIVO</td>\n",
       "      <td>VIVO X30 PRO</td>\n",
       "      <td>湖南</td>\n",
       "      <td>郴州</td>\n",
       "      <td>unkown</td>\n",
       "      <td>tagid:399970be5f6f08fddccaf72113601304;value:9...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        pid  label   brand             model province  city  \\\n",
       "0  ac+1xFYOjvFPT6sUKOPjaA==      0  HUAWEI  HUAWEI NOVA 7 5G       河北    邯郸   \n",
       "1  9RQtNBN4njiXfA60ZbH3Sw==      0  unkown            unkown      黑龙江  齐齐哈尔   \n",
       "2  6x3FhIia6hm3tZTvPPxuLA==      0    OPPO     OPPO RENO6 5G       广东    茂名   \n",
       "3  zHOsTImR/JRdOYHSn1aAMg==      1    VIVO         VIVO Y72T      内蒙古  巴彦淖尔   \n",
       "4  rULwSixw+GLkr1fy/mvMpg==      0    VIVO      VIVO X30 PRO       湖南    郴州   \n",
       "\n",
       "                                            enum_tag  \\\n",
       "0  tagid:4dbc6e1a685a4dfdf9d58c37f2d53c52;time:20...   \n",
       "1                                             unkown   \n",
       "2                                             unkown   \n",
       "3                                             unkown   \n",
       "4                                             unkown   \n",
       "\n",
       "                                         numeric_tag  \n",
       "0  tagid:c03968985782bb457f8124b244c50c30;value:1...  \n",
       "1                                                NaN  \n",
       "2  tagid:d584da8a6f878c250073a365cf5ef73e;value:9...  \n",
       "3  tagid:ecdf085acb4362af9772772f295ea000;value:0...  \n",
       "4  tagid:399970be5f6f08fddccaf72113601304;value:9...  "
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:54:52.207511Z",
     "iopub.status.busy": "2023-08-29T03:54:52.207143Z",
     "iopub.status.idle": "2023-08-29T03:54:52.217698Z",
     "shell.execute_reply": "2023-08-29T03:54:52.216988Z",
     "shell.execute_reply.started": "2023-08-29T03:54:52.207487Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>brand</th>\n",
       "      <th>model</th>\n",
       "      <th>province</th>\n",
       "      <th>city</th>\n",
       "      <th>enum_tag</th>\n",
       "      <th>numeric_tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>++/293LcW9RNSPb72rzSrw==</td>\n",
       "      <td>OPPO</td>\n",
       "      <td>OPPO A5</td>\n",
       "      <td>广西</td>\n",
       "      <td>桂林</td>\n",
       "      <td>tagid:c573867cf7897cad60e6dc8862aceb38;time:20...</td>\n",
       "      <td>tagid:b9431bb43b9032edf7789258c6a9af39;value:9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>++/RlHw/tr5gM03OOUggrg==</td>\n",
       "      <td>HUAWEI</td>\n",
       "      <td>HUAWEI NOVA 6</td>\n",
       "      <td>江西</td>\n",
       "      <td>抚州</td>\n",
       "      <td>unkown</td>\n",
       "      <td>tagid:fa26808877c41781b2d10e5bec855ea9;value:9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>++/fMd264QH48DUFkF269A==</td>\n",
       "      <td>HONOR</td>\n",
       "      <td>CHINA UNICOM HONOR 5A</td>\n",
       "      <td>湖南</td>\n",
       "      <td>株洲</td>\n",
       "      <td>unkown</td>\n",
       "      <td>unkown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>++0KQpW/4rUnCF0YYTP11w==</td>\n",
       "      <td>VIVO</td>\n",
       "      <td>VIVO S6 5G</td>\n",
       "      <td>湖北</td>\n",
       "      <td>孝感</td>\n",
       "      <td>tagid:48c7abbac1adc12fdd2f708643aa648e;time:20...</td>\n",
       "      <td>tagid:4a8f507e6e4be79cd6b1528037928339;value:1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>++0wr5x6b+6xHIQ2kAOovQ==</td>\n",
       "      <td>HUAWEI</td>\n",
       "      <td>DCM</td>\n",
       "      <td>江苏</td>\n",
       "      <td>南京</td>\n",
       "      <td>unkown</td>\n",
       "      <td>unkown</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        pid   brand                  model province city  \\\n",
       "0  ++/293LcW9RNSPb72rzSrw==    OPPO                OPPO A5       广西   桂林   \n",
       "1  ++/RlHw/tr5gM03OOUggrg==  HUAWEI          HUAWEI NOVA 6       江西   抚州   \n",
       "2  ++/fMd264QH48DUFkF269A==   HONOR  CHINA UNICOM HONOR 5A       湖南   株洲   \n",
       "3  ++0KQpW/4rUnCF0YYTP11w==    VIVO             VIVO S6 5G       湖北   孝感   \n",
       "4  ++0wr5x6b+6xHIQ2kAOovQ==  HUAWEI                    DCM       江苏   南京   \n",
       "\n",
       "                                            enum_tag  \\\n",
       "0  tagid:c573867cf7897cad60e6dc8862aceb38;time:20...   \n",
       "1                                             unkown   \n",
       "2                                             unkown   \n",
       "3  tagid:48c7abbac1adc12fdd2f708643aa648e;time:20...   \n",
       "4                                             unkown   \n",
       "\n",
       "                                         numeric_tag  \n",
       "0  tagid:b9431bb43b9032edf7789258c6a9af39;value:9...  \n",
       "1  tagid:fa26808877c41781b2d10e5bec855ea9;value:9...  \n",
       "2                                             unkown  \n",
       "3  tagid:4a8f507e6e4be79cd6b1528037928339;value:1...  \n",
       "4                                             unkown  "
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:54:52.219055Z",
     "iopub.status.busy": "2023-08-29T03:54:52.218629Z",
     "iopub.status.idle": "2023-08-29T03:54:52.819096Z",
     "shell.execute_reply": "2023-08-29T03:54:52.818204Z",
     "shell.execute_reply.started": "2023-08-29T03:54:52.219032Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data['brand_freq_encoding'] = train_data['brand'].map(train_data['brand'].value_counts())\n",
    "test_data['brand_freq_encoding'] = test_data['brand'].map(train_data['brand'].value_counts())\n",
    "\n",
    "train_data['brand_target_encoding'] = train_data['brand'].map(\n",
    "    train_data[train_data['brand'].isin(train_data['brand'].value_counts().index[:50])].groupby('brand')['label'].mean()\n",
    ")\n",
    "test_data['brand_target_encoding'] = test_data['brand'].map(\n",
    "    train_data[train_data['brand'].isin(train_data['brand'].value_counts().index[:50])].groupby('brand')['label'].mean()\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "train_data['model_freq_encoding'] = train_data['model'].map(train_data['model'].value_counts())\n",
    "test_data['model_freq_encoding'] = test_data['model'].map(train_data['model'].value_counts())\n",
    "\n",
    "\n",
    "\n",
    "train_data['province_freq_encoding'] = train_data['province'].map(train_data['province'].value_counts())\n",
    "test_data['province_freq_encoding'] = test_data['province'].map(train_data['province'].value_counts())\n",
    "\n",
    "train_data['province_target_encoding'] = train_data['province'].map(train_data.groupby('province')['label'].mean())\n",
    "test_data['province_target_encoding'] = test_data['province'].map(train_data.groupby('province')['label'].mean())\n",
    "\n",
    "\n",
    "\n",
    "train_data['city_freq_encoding'] = train_data['city'].map(train_data['city'].value_counts())\n",
    "test_data['city_freq_encoding'] = test_data['city'].map(train_data['city'].value_counts())\n",
    "\n",
    "train_data['city_target_encoding'] = train_data['city'].map(train_data.groupby('city')['label'].mean())\n",
    "test_data['city_target_encoding'] = test_data['city'].map(train_data.groupby('city')['label'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:54:52.820520Z",
     "iopub.status.busy": "2023-08-29T03:54:52.820162Z",
     "iopub.status.idle": "2023-08-29T03:54:52.825799Z",
     "shell.execute_reply": "2023-08-29T03:54:52.825029Z",
     "shell.execute_reply.started": "2023-08-29T03:54:52.820495Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tagid:4dbc6e1a685a4dfdf9d58c37f2d53c52;time:20190710|tagid:0d5168923e2e39e622f99f08f7ec73a1;time:20210221|tagid:d579038f347871a088cf927b78991511;time:20200930|tagid:01f4f83ec057ea0fc2b0c93bb7ac07e8;time:20190710|tagid:6e6c0167fada54838a01ef17040721b5;time:20201103|tagid:9a2df192dfa500cbd6cedd313c32b98d;time:20190710|tagid:7626d6aa6b370378e70422ecf4e4f490;time:20190710|tagid:3deb963de78939cb49f564e9874cbe11;time:20200930|tagid:50185ed37d92503b48982c91e3926bc5;time:20210221|tagid:fb19cda708277547190e41d747be556f;time:20190710|tagid:3decda793e84688fded8af9a1da5497d;time:20200930|tagid:5ef5d858650055acd9f9e9ccd4241267;time:20200930|tagid:32c2b9872dfb32a34cd2fe9e192de33d;time:20201218|tagid:550719aa8417810dbdc074b36bad157e;time:20190710|tagid:c573867cf7897cad60e6dc8862aceb38;time:20190710|tagid:48c7abbac1adc12fdd2f708643aa648e;time:20190710|tagid:db77dadb5b5bef4f0abc2e91e55fbc80;time:20190710|tagid:355673944e9d6371bff79e155d32be70;time:20210108|tagid:b69875153aa32c00964c0ee586c65c23;time:20190710|tagid:a7ca056063b132dce4db834be9a06ce1;time:20190710|tagid:16921f18437a960dfe8327c8dcee78f6;time:20210221|tagid:929bb21ce7e19999860ab238bad7e6e1;time:20200930|tagid:2adf4dac009b8e1a20211a6ad77aa08d;time:20190710|tagid:a28f38698dd3f84c9064af6996bbb135;time:20190710|tagid:6f93ca869ca2ebbf24b3b543e0cead9d;time:20190710|tagid:03b5d585bab62d9392beed3b5cd1f7ee;time:20210221|tagid:8fc8570958bad4bef511376b1efe3552;time:20200930|tagid:bc7f3ac4420b87e4aa9b892f6abe1dfb;time:20201025|tagid:65d52e4eb0c739e9e8b180c8a6342e3b;time:20200930|tagid:5620f4af8b9f7268a5380dd32d99c168;time:20190710|tagid:1d1a4091f068f0a8895cb206b486f1f6;time:20200930|tagid:cb99962d46ed57f98ea0fd9940cdd90d;time:20190710|tagid:9a794bd45460b0667f532cdcd3e170fa;time:20190710|tagid:31dbde0bac7b5e8f28f05ae3489ecbfe;time:20210221|tagid:cd06998f2238754f7e1f4646e2916c50;time:20210221|tagid:60b59be29b52719e27ba731fa50aeade;time:20190710|tagid:9a02e268b83900770ee878c394cff659;time:20200930|tagid:d42c7a6fd2cef6c9add34e3118372f08;time:20210221'"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data['enum_tag'].iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:54:52.827916Z",
     "iopub.status.busy": "2023-08-29T03:54:52.827588Z",
     "iopub.status.idle": "2023-08-29T03:55:22.156116Z",
     "shell.execute_reply": "2023-08-29T03:55:22.155285Z",
     "shell.execute_reply.started": "2023-08-29T03:54:52.827888Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "import re\n",
    "split_rule = re.compile(r'[:;]')\n",
    "\n",
    "def split_enum_tag(tag):\n",
    "    if tag == \"unkown\":\n",
    "        return []\n",
    "    \n",
    "    tags = tag.split('|')\n",
    "    tags = [split_rule.split(x) for x in tags]\n",
    "    tags = [[x[1], x[-1]] for x in tags]\n",
    "    return np.array(tags)\n",
    "\n",
    "train_data['enum_tag'] = train_data['enum_tag'].apply(split_enum_tag)\n",
    "test_data['enum_tag'] = test_data['enum_tag'].apply(split_enum_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:55:22.157794Z",
     "iopub.status.busy": "2023-08-29T03:55:22.157239Z",
     "iopub.status.idle": "2023-08-29T03:57:23.124683Z",
     "shell.execute_reply": "2023-08-29T03:57:23.123809Z",
     "shell.execute_reply.started": "2023-08-29T03:55:22.157768Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "def feat_enum_tag(tag):\n",
    "    if len(tag) == 0:\n",
    "        return [0, 0, 0, 0, 0]\n",
    "    \n",
    "    return [\n",
    "        tag.shape[0],\n",
    "        len(set(tag[:, 0])),\n",
    "        len(set(tag[:, 1])),\n",
    "        len(set([x[:4] for x in tag[:, 1]])),\n",
    "        len(set([x[4:6] for x in tag[:, 1]]))\n",
    "    ]\n",
    "\n",
    "train_enum_tag_feat = pd.DataFrame(train_data['enum_tag'].apply(feat_enum_tag))\n",
    "train_enum_tag_feat = train_enum_tag_feat['enum_tag'].apply(pd.Series)\n",
    "\n",
    "test_enum_tag_feat = pd.DataFrame(test_data['enum_tag'].apply(feat_enum_tag))\n",
    "test_enum_tag_feat = test_enum_tag_feat['enum_tag'].apply(pd.Series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:57:23.130853Z",
     "iopub.status.busy": "2023-08-29T03:57:23.130573Z",
     "iopub.status.idle": "2023-08-29T03:57:23.136270Z",
     "shell.execute_reply": "2023-08-29T03:57:23.135621Z",
     "shell.execute_reply.started": "2023-08-29T03:57:23.130829Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tagid:c03968985782bb457f8124b244c50c30;value:100.0;time:20230208|tagid:e0ab4a14e341d4b302130e22529dfae4;value:98.0;time:20230115|tagid:057d3f523e49b309287a13d07435c437;value:95.0;time:20230115|tagid:61f79f744f3bf951a5a3485c34d93b12;value:90.0;time:20221107|tagid:1fc94cd2e265eaefdd6fc971a7c25f73;value:93.0;time:20221204|tagid:61c24bcde90c9a0be92836e1867674ce;value:98.0;time:20230221|tagid:bb182b2a72b4ab11e4c596c989cbaece;value:98.0;time:20230221|tagid:8cbae4fd77102363c0ccd26283beecf3;value:86.814;time:20230213|tagid:7fe9fcb213a9c3557ebe9bc494f2e25a;value:98.0;time:20220805|tagid:e08184514ec8217420f186391b384ad3;value:98.0;time:20230115|tagid:76e9bd851c883233734394abac490970;value:100.0;time:20230221|tagid:dec9a606a36a1add38c22cc94a1406ac;value:100.0;time:20230221|tagid:860c50da17eb1409b14179111eeac900;value:0.0;time:20230326|tagid:9c982c7f65b36e62290f27d5924da73f;value:96.0;time:20221204|tagid:e7c88f84a4ec23f8583d5f9b234355c0;value:96.0;time:20221204|tagid:e43be3ed3e17d8dad04c41e01e714bbb;value:97.0;time:20230221|tagid:69df891c9f2a5ca71e52b03077dbd1da;value:100.0;time:20230115|tagid:a69e633c7ced16640f4937d3bdd75913;value:88.591;time:20230213|tagid:d8dddc07520c207817fbd9bc8ae4641c;value:76.524;time:20230213|tagid:8e931eec6ded988fd6c482f3963d8e25;value:97.0;time:20230115|tagid:95a8c7cc3825f5192d2a54cd492e881c;value:0.0;time:20230329|tagid:07f6909018e101377000eb85e2d58b34;value:100.0;time:20221204|tagid:56d0a0fe2c4a155ce2b0b89fba7fa6ae;value:100.0;time:20230221|tagid:0082dfc7a673cf02018c749a79289031;value:100.0;time:20220723|tagid:4a8f507e6e4be79cd6b1528037928339;value:100.0;time:20221204|tagid:ff40675c6c45dac9ca5b414905ac6a40;value:99.0;time:20230221|tagid:c2124f78acf3f73ef83c448b787a46fd;value:100.0;time:20220911|tagid:fa67357136f5ef27397b094eb179ab58;value:92.0;time:20230115|tagid:45b20ba54b71eb4599540cb1fdf665c9;value:100.0;time:20230221|tagid:c73c3ad0500fb72251855c0cdef9fa34;value:99.0;time:20230208'"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data['numeric_tag'].iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:57:23.139463Z",
     "iopub.status.busy": "2023-08-29T03:57:23.139060Z",
     "iopub.status.idle": "2023-08-29T03:57:45.592310Z",
     "shell.execute_reply": "2023-08-29T03:57:45.591457Z",
     "shell.execute_reply.started": "2023-08-29T03:57:23.139439Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "def split_numeric_tag(tag):\n",
    "    if str(tag) in ['nan', 'unkown']:\n",
    "        return []\n",
    "    \n",
    "    tags = tag.split('|')\n",
    "    tags = [split_rule.split(x) for x in tags]\n",
    "    tags = [[x[1], float(x[-3]), x[-1]] for x in tags]\n",
    "    return np.array(tags)\n",
    "\n",
    "train_data['numeric_tag'] = train_data['numeric_tag'].apply(split_numeric_tag)\n",
    "test_data['numeric_tag'] = test_data['numeric_tag'].apply(split_numeric_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:57:45.593844Z",
     "iopub.status.busy": "2023-08-29T03:57:45.593468Z",
     "iopub.status.idle": "2023-08-29T03:59:56.146677Z",
     "shell.execute_reply": "2023-08-29T03:59:56.145819Z",
     "shell.execute_reply.started": "2023-08-29T03:57:45.593819Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "def feat_numeric_tag(tag):\n",
    "    if len(tag) == 0:\n",
    "        return [0, 0, 0, 0, 0, 0, 0, 0]\n",
    "    \n",
    "    return [\n",
    "        tag.shape[0],\n",
    "        len(set(tag[:, 0])),\n",
    "        len(set(tag[:, -1])),\n",
    "        tag[:, 1].astype(float).mean(),\n",
    "        tag[:, 1].astype(float).max(),\n",
    "        tag[:, 1].astype(float).min(),\n",
    "\n",
    "        len(set([x[:4] for x in tag[:, -1]])),\n",
    "        len(set([x[4:6] for x in tag[:, -1]]))\n",
    "    ]\n",
    "\n",
    "train_numeric_tag_feat = pd.DataFrame(train_data['numeric_tag'].apply(feat_numeric_tag))\n",
    "train_numeric_tag_feat = train_numeric_tag_feat['numeric_tag'].apply(pd.Series)\n",
    "\n",
    "test_numeric_tag_feat = pd.DataFrame(test_data['numeric_tag'].apply(feat_numeric_tag))\n",
    "test_numeric_tag_feat = test_numeric_tag_feat['numeric_tag'].apply(pd.Series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:59:56.153710Z",
     "iopub.status.busy": "2023-08-29T03:59:56.153306Z",
     "iopub.status.idle": "2023-08-29T03:59:56.158558Z",
     "shell.execute_reply": "2023-08-29T03:59:56.157781Z",
     "shell.execute_reply.started": "2023-08-29T03:59:56.153685Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((209470, 15), (209470, 8), (209470, 5))"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.shape, train_numeric_tag_feat.shape, train_enum_tag_feat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:59:56.159789Z",
     "iopub.status.busy": "2023-08-29T03:59:56.159469Z",
     "iopub.status.idle": "2023-08-29T03:59:56.164511Z",
     "shell.execute_reply": "2023-08-29T03:59:56.163888Z",
     "shell.execute_reply.started": "2023-08-29T03:59:56.159765Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((490530, 14), (490530, 8), (490530, 5))"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.shape, test_numeric_tag_feat.shape, test_enum_tag_feat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T03:59:56.172895Z",
     "iopub.status.busy": "2023-08-29T03:59:56.172201Z",
     "iopub.status.idle": "2023-08-29T03:59:56.177887Z",
     "shell.execute_reply": "2023-08-29T03:59:56.177174Z",
     "shell.execute_reply.started": "2023-08-29T03:59:56.172871Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['pid', 'brand', 'model', 'province', 'city', 'enum_tag', 'numeric_tag',\n",
       "       'brand_freq_encoding', 'brand_target_encoding', 'model_freq_encoding',\n",
       "       'province_freq_encoding', 'province_target_encoding',\n",
       "       'city_freq_encoding', 'city_target_encoding'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T04:11:04.819286Z",
     "iopub.status.busy": "2023-08-29T04:11:04.818369Z",
     "iopub.status.idle": "2023-08-29T04:11:07.737636Z",
     "shell.execute_reply": "2023-08-29T04:11:07.736417Z",
     "shell.execute_reply.started": "2023-08-29T04:11:04.819255Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>DecisionTreeClassifier()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">DecisionTreeClassifier</label><div class=\"sk-toggleable__content\"><pre>DecisionTreeClassifier()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "DecisionTreeClassifier()"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "model = DecisionTreeClassifier()\n",
    "model.fit(\n",
    "    np.hstack(\n",
    "        [train_data.drop(['pid', 'brand', 'label','model', 'province', 'city', 'enum_tag', 'numeric_tag'], axis=1).fillna(0).values, \n",
    "        train_numeric_tag_feat.values, \n",
    "        train_enum_tag_feat.values]\n",
    "    ),\n",
    "    train_data['label']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T04:11:09.149054Z",
     "iopub.status.busy": "2023-08-29T04:11:09.148162Z",
     "iopub.status.idle": "2023-08-29T04:11:09.404917Z",
     "shell.execute_reply": "2023-08-29T04:11:09.404032Z",
     "shell.execute_reply.started": "2023-08-29T04:11:09.149023Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_pred = model.predict_proba(\n",
    "    np.hstack(\n",
    "        [test_data.drop(['pid', 'brand','model', 'province', 'city', 'enum_tag', 'numeric_tag'], axis=1).fillna(0).values, \n",
    "        test_numeric_tag_feat.values, \n",
    "        test_enum_tag_feat.values]\n",
    "    )\n",
    ")[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T04:11:10.573272Z",
     "iopub.status.busy": "2023-08-29T04:11:10.572534Z",
     "iopub.status.idle": "2023-08-29T04:11:10.920398Z",
     "shell.execute_reply": "2023-08-29T04:11:10.919585Z",
     "shell.execute_reply.started": "2023-08-29T04:11:10.573240Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>++/293LcW9RNSPb72rzSrw==</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>++/RlHw/tr5gM03OOUggrg==</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>++/fMd264QH48DUFkF269A==</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>++0KQpW/4rUnCF0YYTP11w==</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>++0wr5x6b+6xHIQ2kAOovQ==</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490525</th>\n",
       "      <td>zzxnCLsCCwWbD/77zz/5VQ==</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490526</th>\n",
       "      <td>zzyr6Emr9GGgbcmPbPFndg==</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490527</th>\n",
       "      <td>zzyyHiqkSJdenstYOiOyRg==</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490528</th>\n",
       "      <td>zzz43AAVMvD28vfvRZIVIA==</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490529</th>\n",
       "      <td>zzzhMMPlSKgrpyYketeTDQ==</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>490530 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             pid  label\n",
       "0       ++/293LcW9RNSPb72rzSrw==      0\n",
       "1       ++/RlHw/tr5gM03OOUggrg==      0\n",
       "2       ++/fMd264QH48DUFkF269A==      0\n",
       "3       ++0KQpW/4rUnCF0YYTP11w==      0\n",
       "4       ++0wr5x6b+6xHIQ2kAOovQ==      0\n",
       "...                          ...    ...\n",
       "490525  zzxnCLsCCwWbD/77zz/5VQ==      0\n",
       "490526  zzyr6Emr9GGgbcmPbPFndg==      0\n",
       "490527  zzyyHiqkSJdenstYOiOyRg==      0\n",
       "490528  zzz43AAVMvD28vfvRZIVIA==      0\n",
       "490529  zzzhMMPlSKgrpyYketeTDQ==      0\n",
       "\n",
       "[490530 rows x 2 columns]"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(\"提交示例.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-29T04:11:12.153590Z",
     "iopub.status.busy": "2023-08-29T04:11:12.152652Z",
     "iopub.status.idle": "2023-08-29T04:11:12.821101Z",
     "shell.execute_reply": "2023-08-29T04:11:12.820231Z",
     "shell.execute_reply.started": "2023-08-29T04:11:12.153560Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.DataFrame({\n",
    "    'pid': test_data['pid'],\n",
    "    'label': test_pred.round(3)\n",
    "}).to_csv('submit.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "py35-paddle1.2.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
